#!/usr/bin/python
#-*-coding:utf-8-*-
# 简易小说采集爬虫
# Author: Lukin<mylukin@gmail.com>
# Date  : 2008-09-25
import re
import urllib2
import sqlite3
import os.path
import string
import time
import socket
from download import httpfetch

part="Html/Book/5/5820/Index.shtml"
headers = {"Accept": "*/*", "Referer": "http://www.16kbook.com/Book/ShowBookTop.aspx", "User-Agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)"}
baseurl = "http://www.16kbook.com/"
data = None
bid = None
socket.setdefaulttimeout(20)
#建立数据库
def createDatabase(bid):
    dbname = './' + bid + '.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    # 创建url列表存放表
    c.execute('SELECT * FROM sqlite_master WHERE name = \'List\';')
    if c.fetchall():
        c.execute('DROP TABLE \'List\'')
        conn.commit()
    c.execute('''CREATE TABLE IF NOT EXISTS List (  Cid INTEGER  PRIMARY KEY, Bid INTEGER, Title TEXT(60) );''')
    c.execute('''CREATE TABLE IF NOT EXISTS Book (  Cid INTEGER  PRIMARY KEY, Title TEXT ,text text );''')
    # 事物提交
    conn.commit()
    c.close()
#读取页面内容
def read_url(part="Html/Book/5/5820/Index.shtml"):
    bbid = string.atoi(part,10)//1000
    url = baseurl + "Html/Book/" + str(bbid) + "/" + part + "/Index.shtml"
    req = httpfetch(url,headers)    
    # 更新记录
    #updateOneUrl(url,R)
    # 返回结果
    return req.decode('gb2312','ignore').encode('utf-8')


#读取文章页面
def read_content_url(bid,cid):
    bbid = string.atoi(bid,10)//1000
    url = baseurl + "Html/Book/" + str(bbid) + "/" + str(bid) + "/" + str(cid) + ".shtml"
    req = httpfetch(url,headers)    
    # 更新记录
    #updateOneUrl(url,R)
    # 返回结果
    return req.decode('gb2312','ignore').encode('utf-8')

#分析列表页函数
def parseData(html,bid):
    dbname = './' + bid + '.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    i = 0
    names = re.findall(r'<li><a href="([0-9]+).shtml"><FONT color=#000000>(.+)</FONT></a></li>',html)
    for name in names:
        if cmp(name[1], '本章为防采集章节') <> 0 :
            i = i + 1
            c.execute('SELECT * FROM List WHERE Bid = ?;',(name[0],))
            if c.fetchone():
                print name[0]
                #c.execute('UPDATE Book (Bid,Title) VALUES (?,?);',(name[0],name[1]))
            else:
                c.execute('INSERT INTO List (Bid,Title) VALUES (?,?);',(name[0],name[1]))
    conn.commit(); c.close()
    print i
    return names

#分析文章内容
def parse_content(html,bid,cid):
    dbname = './' + bid + '.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    content = sect(html,'(<script src=\'/js/readadend.js\'></script>)','(<div align="center"><a href=)')
    title = re.findall(r'<span class="newstitle">(.+)</span>',html)
    if content != None and title != None:
        c.execute('SELECT * FROM Book where Cid = ?',(cid,))
        if c.fetchone():
           c.execute('DELETE FROM Book where Cid = ?',(cid,))
           c.execute('INSERT INTO Book (Cid,Title,text) VALUES (?,?,?);',(cid,title[0],content,))
        else:
           c.execute('INSERT INTO Book (Cid,Title,text) VALUES (?,?,?) ;',(cid,title[0],content,))
           conn.commit(); c.close()
           print cid
           return content
    #c.execute('INSERT INTO Content (Cid,Title,text) VALUES (?,?,?);',(cid,title,content))
    

# html代码截取函数
def sect(html,start,end,cls=''):
    if len(html)==0 : return ;
    # 正则表达式截取
    if start[:1]==chr(40) and start[-1:]==chr(41) and end[:1]==chr(40) and end[-1:]==chr(41) :
        reHTML = re.search(start + '(.*?)' + end,html,re.I)
        if reHTML == None : return 
        reHTML = reHTML.group()
        intStart = re.search(start,reHTML,re.I).end()
        intEnd = re.search(end,reHTML,re.I).start()
        R = reHTML[intStart:intEnd]
    # 字符串截取
    else :
        # 取得开始字符串的位置
        intStart = html.lower().find(start.lower())
        # 如果搜索不到开始字符串，则直接返回空
        if intStart == -1 : return 
        # 取得结束字符串的位置
        intEnd = html[intStart+len(start):].lower().find(end.lower())
        # 如果搜索不到结束字符串，也返回为空
        if intEnd == -1 : return 
        # 开始和结束字符串都有了，可以开始截取了
        R = html[intStart+len(start):intStart+intEnd+len(start)]
    # 清理内容
    if cls != '' :
        R = clear(R,cls)
    # 返回截取的字符
    return R

def clear(html,regexs):
    if regexs == '' : return html
    for regex in regexs.split(chr(10)):
        regex = regex.strip()
        if regex != '' :
            if regex[:1]==chr(40) and regex[-1:]==chr(41):
                html = re.sub(regex,'',html,re.I|re.S)
            else :
                html = html.replace(regex,'')
    return html

#获取章节列表函数
def get_list(bid):
    dbname = './' + bid + '.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    c.execute('SELECT Bid,Cid FROM List')
    return c.fetchall()
    
def get_content(bid,cid):
    dbname = './' + bid + '.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    c.execute('SELECT * FROM Book WHERE Cid = ?',(cid,))
    return c.fetchone()
    
def make_book(bid):
    if (bid == None):
        bid = raw_input('please input book id:')
    bid = str(bid)
    createDatabase(bid)
    html = read_url(bid)
    html1 = parseData(html,bid)
    contentlists = get_list(bid)
    for contentlist in contentlists:
        G = get_content(bid,contentlist[1])
        if G == None:
            parse_content(read_content_url(bid,contentlist[0]),bid,contentlist[1])
            print '成功读取:'.decode('utf-8','ignore').encode('gbk') + str(contentlist[1])        
        elif G[2] == None:
            parse_content(read_content_url(bid,contentlist[0]),bid,contentlist[1])
            print '成功读取:'.decode('utf-8','ignore').encode('gbk') + str(contentlist[1])
        elif len(G[2]) < 100:
            parse_content(read_content_url(bid,contentlist[0]),bid,contentlist[1])
            print '成功读取:'.decode('utf-8','ignore').encode('gbk') + str(contentlist[1])
        else:
            print '不需要读取:'.decode('utf-8','ignore').encode('gbk') + str(contentlist[1])
    return True
            
def get_book_list():
    dbname = './list.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    c.execute('SELECT * FROM List')
    return c.fetchall()

def update_book_list(bid):
    dbname = './list.db'
    conn = sqlite3.connect(os.path.abspath(dbname))
    conn.text_factory = str
    c = conn.cursor()
    c.execute('UPDATE List set get = ?,finished = ? WHERE Bid = ? ;',('1','1',bid))
    conn.commit()
    c.close()
    return True


for list in get_book_list():
    if list[3] == None:
        print '开始下载'.decode('utf-8','ignore').encode('gbk') + str(list[1])
        make_book(list[1])
        print '成功下载:'.decode('utf-8','ignore').encode('gbk') + str(list[1])
        update_book_list(list[1])
    else:
        print '不需要下载:'.decode('utf-8','ignore').encode('gbk') + str(list[1])

   
    
